package com.spider.web.url;

import java.io.*;
import java.net.*;
import java.util.Enumeration;
import java.util.HashSet;
import java.util.Hashtable;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.Map.Entry;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.commons.httpclient.*;
import org.apache.commons.httpclient.methods.GetMethod;
import org.apache.commons.httpclient.methods.PostMethod;
import org.apache.commons.httpclient.params.HttpMethodParams;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.log4j.Logger;
import org.htmlparser.beans.StringBean;

import com.car.file.FileUtil;

/**
 * @class MyHttpClient.class 解析html文件
 * @method getHtmlSourceCode(String url);取到html源文件。以String 格式 返回.
 * @method getHtmlAsReader(String url) ;取到html源文件。以 BufferReader 文件流格式 返回.
 * @method getDescription(String page);通过正则表达式,获取description的时候 ; page可以是String
 *         or 文件流toString()。
 * @author pc221
 * 
 */

public final class MyHttpClient {

	public static Logger logger = Logger.getLogger(MyHttpClient.class);

	public static PostMethod getPostMethod(String url, String[] textname,
			String[] params, String charset) throws IOException, IOException {
		// 1.生成httpclient实例

		HttpClient httpClient = new HttpClient();

		// 2.post生成Post 方法

		PostMethod postMethod = new PostMethod(url);

		// 参数
		NameValuePair[] data = new NameValuePair[params.length];

		for (int x = 0; x < params.length; x++) {
			data[x] = new NameValuePair(textname[x], params[x]);
			logger.info("设置第" + (x + 1) + "参数{[key=" + textname[x] + "][value="
					+ params[x] + "]}");
			System.out.println("设置第" + (x + 1) + "参数{[key=" + textname[x]
					+ "][value=" + params[x] + "]}");

		}
		logger.info(data);

		// 3设置编码

		httpClient.getParams().setParameter(
				HttpMethodParams.HTTP_CONTENT_CHARSET, charset);

		// 4提交请求
		postMethod.setRequestBody(data);

		/* 3.执行 HTTP GET 请求 */

		int statusCode = httpClient.executeMethod(postMethod);
		// 判断访问的状态码
		if (statusCode != HttpStatus.SC_OK) {
			System.err.println("Method failed: " + postMethod.getStatusLine());

		}

		// 返回结果
		System.out.println(postMethod.getResponseBodyAsString());
		System.out.println(postMethod.getResponseBodyAsStream());
		// 301 or 302 继续提交
		return postMethod;

	}

	public static PostMethod getPostMethod(String url,
			Map<String, String[]> map, String charset) throws IOException,
			IOException {

		// 1.生成httpclient实例

		HttpClient httpClient = new HttpClient();

		// 2.post生成Post 方法

		PostMethod postMethod = new PostMethod(url);

		// 参数
		NameValuePair[] data = new NameValuePair[map.size()];
		int index = 0;
		Set<String> keyMap = map.keySet();
		for (String mapKey : keyMap) {
			if (StringUtils.isEmpty(mapKey))
				continue;
			String[] values = (String[]) map.get(mapKey);
			if (values == null || values.length == 0)
				continue;
			data[index] = new NameValuePair(mapKey, values[0]);
			logger.info("设置第" + (index + 1) + "参数{[key=" + mapKey + "][value="
					+ values[0] + "]}");
			System.out.println("设置第" + (index + 1) + "参数{[key=" + mapKey
					+ "][value=" + values[0] + "]}");
			index++;

		}
		logger.info(data);

		// 3设置编码

		httpClient.getParams().setParameter(
				HttpMethodParams.HTTP_CONTENT_CHARSET, charset);

		// 4提交请求
		postMethod.setRequestBody(data);

		/* 3.执行 HTTP GET 请求 */

		int statusCode = httpClient.executeMethod(postMethod);
		// 判断访问的状态码
		if (statusCode != HttpStatus.SC_OK) {
			System.err.println("Method failed: " + postMethod.getStatusLine());

		}

		// 返回结果
		System.out.println(postMethod.getResponseBodyAsString());
		System.out.println(postMethod.getResponseBodyAsStream());

		// 301 or 302 继续提交
		return postMethod;

	}

	/**
	 * httpClient 封装,调用此方法后，需要写程序释放 Method ; Method.releaseConnection()
	 * GetMethod(String url);
	 * 
	 * @param url
	 * @return
	 * @throws HttpException
	 * @throws IOException
	 */

	public static GetMethod getPostMethod(String url) throws IOException,
			IOException {
		/* 1.生成 HttpClinet 对象并设置参数 */
		HttpClient httpClient = new HttpClient();
		// 设置 Http 连接超时 5s
		httpClient.getHttpConnectionManager().getParams()
				.setConnectionTimeout(50000);

		httpClient.getParams().setParameter(
				HttpMethodParams.HTTP_CONTENT_CHARSET, "gb2312");

		/* 2.生成 GetMethod 对象并设置参数 */
		GetMethod getMethod = new GetMethod(url);

		// 设置 get 请求超时 20s
		getMethod.getParams().setParameter(HttpMethodParams.SO_TIMEOUT, 20000);
		// 设置请求重试处理
		getMethod.getParams().setParameter(HttpMethodParams.RETRY_HANDLER,
				new DefaultHttpMethodRetryHandler());

		/* 3.执行 HTTP GET 请求 */

		int statusCode = httpClient.executeMethod(getMethod);
		// 判断访问的状态码
		if (statusCode != HttpStatus.SC_OK) {
			System.err.println("Method failed: " + getMethod.getStatusLine());

		}

		/* 4.处理 HTTP 响应内容 */
		// byte[] responseBody = getMethod.getResponseBody();// 读取为字节数组
		// InputStream in = getMethod.getResponseBodyAsStream();
		return getMethod;

	}

	/**
	 * 取到传入URL页面的HTML源文件。
	 * 
	 * @param 参数
	 *            url
	 * @return
	 * @throws HttpException
	 * @throws IOException
	 */

	public static String getHtmlSourceCode(String url) throws HttpException,
			IOException {
		GetMethod getMethod = null;
		try {
			getMethod = getPostMethod(url);
			byte[] responseBody = getMethod.getResponseBody();
			String htmlsrc = new String(responseBody);
			System.out.println(htmlsrc);
			return htmlsrc;
		} finally {
			// 释放连接
			getMethod.releaseConnection();

		}

	}

	/**
	 * 取到传入filepath的文件流。
	 * 
	 * @param 参数
	 * 
	 * @return
	 * @throws IOException
	 */

	public static BufferedReader getHtmlAsReader(String filepath)
			throws IOException {

		BufferedReader reader = new BufferedReader(new FileReader(new File(
				filepath)));

		return reader;
	}

	/**
	 * 取到传入URL页面的HTML文件流。
	 * 
	 * @param 参数
	 *            url
	 * @return
	 * @throws HttpException
	 * @throws IOException
	 */

	public static BufferedReader getHtmlAsReader(String url, String filepath)
			throws HttpException, IOException {
		GetMethod getMethod = getPostMethod(url);
		try {
			InputStream in = getMethod.getResponseBodyAsStream();
			BufferedReader reader = new BufferedReader(new FileReader(new File(
					filepath)));
			while (reader.ready()) {
				System.out.println("MyHttpClient.class.getHtmlAsReader(url):"
						+ reader.readLine());
			}
			return reader;
		} finally {
			// 释放连接
			getMethod.releaseConnection();

		}
	}

	public static BufferedReader getHtmlAsReader_tmp(String url)
			throws HttpException, IOException {
		GetMethod getMethod = getPostMethod(url);
		try {
			InputStream in = getMethod.getResponseBodyAsStream();
			BufferedReader reader = new BufferedReader(new InputStreamReader(
					in, "UTF-8"));
			while (reader.ready()) {
				System.out.println("MyHttpClient.class.getHtmlAsReader(url):"
						+ reader.readLine());
				 
			}
			return reader;
		} finally {
			// 释放连接
			getMethod.releaseConnection();

		}
	}

	public static void DownLoadPages(String urlStr, String outPath) {
		int chByte = 0;
		URL url = null;
		HttpURLConnection httpConn = null;
		InputStream in = null;
		FileOutputStream out = null;

		try {
			url = new URL(urlStr);
			httpConn = (HttpURLConnection) url.openConnection();
			HttpURLConnection.setFollowRedirects(true);
			httpConn.setRequestMethod("GET");
			httpConn.setRequestProperty("User-Agent",
					"Mozilla/4.0 (compatible; MSIE 6.0; Windows xp)");
			System.out.print(httpConn.getResponseMessage());
			 
			logger.info(httpConn.getResponseMessage());
			logger.info(httpConn.getContent());
			System.out.print(httpConn.getResponseCode());
			in = httpConn.getInputStream();
			out = new FileOutputStream(new File(outPath));

			chByte = in.read();
			while (chByte != -1) {
				out.write(chByte);
				chByte = in.read();
			}
		} catch (MalformedURLException e) {
			e.printStackTrace();
		} catch (IOException e) {
			e.printStackTrace();
		} finally {
			try {
				out.close();
				in.close();
				httpConn.disconnect();
			} catch (Exception ex) {
				ex.printStackTrace();
			}
		}
	}

	public static void DownLoadPagesPay(String urlStr, String outPath) {
		int chByte = 0;
		URL url = null;
		HttpURLConnection httpConn = null;
		InputStream in = null;
		FileOutputStream out = null;

		try {
			url = new URL(urlStr);
			httpConn = (HttpURLConnection) url.openConnection();
			HttpURLConnection.setFollowRedirects(true);
			httpConn.setRequestMethod("GET");
			httpConn.setRequestProperty("User-Agent",
					"Mozilla/4.0 (compatible; MSIE 6.0; Windows xp)");
			//System.out.print(httpConn.getResponseMessage()
					//+ httpConn.getResponseCode());
			in = httpConn.getInputStream();
			out = new FileOutputStream(new File(outPath));

			chByte = in.read();
			while (chByte != -1) {
				out.write(chByte);
				chByte = in.read();
			}

		} catch (MalformedURLException e) {
			e.printStackTrace();
		} catch (IOException e) {
			e.printStackTrace();
		} finally {
			try {
				out.close();
				in.close();
				httpConn.disconnect();
			} catch (Exception ex) {
				ex.printStackTrace();
			}
		}
		 
	}

	/**
	 * 通过正则表达式,获取description的时候 ;
	 * 
	 * @param page
	 * @return
	 */
	// <meta http-equiv="Content-Type" content="text/html; charset=gb2312" />
	public static String getDescription(String page) {
		String description = null;
		// 过滤<meta
		Pattern pattern = Pattern.compile(
				"(<meta)(.*)name *= *[\"\'] *description *[\"\'](.*)(/>)",
				Pattern.CASE_INSENSITIVE);
		Matcher matcher = pattern.matcher(page);

		// boolean found = false;//flag indicating whether the page has a
		// description.
		String resultStr = null;
		while (matcher.find()) {
			resultStr = matcher.group();
			// 取到 内容
			Pattern patternDesc = Pattern.compile(
					"(charset *= *[\"\'])(.*)([\"\'])",
					Pattern.CASE_INSENSITIVE);
			Matcher matcherDesc = patternDesc.matcher(resultStr);
			while (matcherDesc.find()) {
				description = matcherDesc.group(2);
				System.out.println(description);
			}
			// found = true;
		}

		// If no description found, then set title as its description
		/*
		 * if(!found){ description = title; }
		 */

		return description;
	}

	public static void postMethod(String url) throws IOException {
		String cookie = "http://www.qianlijob.com";
		HttpURLConnection urlConn = (HttpURLConnection) (new URL(url)
				.openConnection());
		urlConn.addRequestProperty("Cookie", cookie);
		urlConn.setRequestMethod("POST");
		urlConn.setRequestProperty("User-Agent",
				"Mozilla/4.0 (compatible; MSIE 6.0; Windows 2000)");
		urlConn.setFollowRedirects(true);
		urlConn.setDoOutput(true); // 需要向服务器写数据
		urlConn.setDoInput(true); //
		urlConn.setUseCaches(false); // 获得服务器最新的信息
		urlConn.setAllowUserInteraction(false);
		urlConn.setRequestProperty("Content-Type",
				"application/x-www-form-urlencoded");
		urlConn.setRequestProperty("Content-Language", "en-US");
		urlConn.setRequestProperty("Content-Length", "" + "data.length()");

		DataOutputStream outStream = new DataOutputStream(
				urlConn.getOutputStream());
		outStream.writeBytes("data");
		outStream.flush();
		outStream.close();

		cookie = urlConn.getHeaderField("Set-Cookie");
		BufferedReader br = new BufferedReader(new InputStreamReader(
				urlConn.getInputStream(), "gb2312"));
	}

	private static String getTag(String pattern, String str) {
		System.out.println(str);
		Pattern sp = Pattern.compile(pattern);
		Matcher matcher = sp.matcher(str);
		while (matcher.find()) {
			for (int i = 0; i < matcher.groupCount(); i++) {
				System.out.println("  html 正则表达式解析 " + matcher.group(i));
				
			}
		}
		return "";
	}

	private static String convert(String str) {
		StringBuffer sb = new StringBuffer(1024);
		sb.setLength(0);
		for (int i = 0; i < str.length(); i++) {
			char c = str.charAt(i);
			if (c > 255) {
				sb.append("\\u");
				int j = (c >>> 8);
				String tmp = Integer.toHexString(j);
				if (tmp.length() == 1) {
					sb.append("0");
				}
				sb.append(tmp);
				j = (c & 0xFF);
				tmp = Integer.toHexString(j);
				if (tmp.length() == 1) {
					sb.append("0");
				}
				sb.append(tmp);
			} else {
				sb.append(c);
			}
		}
		return (new String(sb));
	}

	public static void main(String[] args) {/*
											 * 
											 * // String url = //
											 * "http://91.usst.edu.cn/FaxList.asp?ntype=企业招聘&ctype=传真招聘&TCareerID=2987"
											 * ; String url1 =
											 * "http://www.cn.alipay-inc.com/souren?k="
											 * ;// // try { // url = //
											 * java.net.URLEncoder.encode(
											 * "http://91.usst.edu.cn/FaxList.asp?ntype=企业招聘&ctype=传真招聘&TCareerID=2987"
											 * ,"UTF-8"); // } catch
											 * (UnsupportedEncodingException e1)
											 * { // // TODO Auto-generated catch
											 * block // e1.printStackTrace(); //
											 * } try { //url =
											 * MyHttpClient.convert(url);
											 * GetMethod method =
											 * MyHttpClient.getPostMethod(url1);
											 * String content =
											 * method.getResponseBodyAsString();
											 * //getTag(
											 * "<a[^>]*>(?<text>[^<].*?)</a>",
											 * content);
											 * System.out.println(content);
											 * System
											 * .out.println(method.getStatusText
											 * ());
											 * 
											 * 
											 * DownLoadPages( url1,
											 * "c://apliay.html");
											 * 
											 * } catch (IOException e) { // TODO
											 * Auto-generated catch block
											 * e.printStackTrace(); }
											 */

		 //parsePay();
	      //测试
		/*try {
			//InstallCert.main(new String[]{} );
		} catch (Exception e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		String BASEURL = " ";//
		DownLoadPages( BASEURL,
				   "c://t.html");*/
		/*String pageKey = "&page=";
		String baseDir="d://pay//";
		Object xing="zhang";
		String ming="l";
		 try {
			parse(  xing,  ming,  BASEURL,  baseDir,  pageKey) ;
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}*/
   // down();		
		String url="https://is.taobao.org/souren/default.aspx?b=全部&query=%e6%9d%8e%e7%90%aa";
		DownLoadPages( url,
		   "c://t.html");
	}
   public static void parsebaba(){
	   String BASEURL = " ";//
		DownLoadPages( BASEURL,
				   "c://t.html");
   }
	private static void parsePay() {
		String BASEURL = "";//
		 
		String pageKey = "&page=";
		String baseDir=" ";
		// zhang&page1=2
		try {

			// GetMethod method = MyHttpClient.getPostMethod(url1);
			// String content = method.getResponseBodyAsString();
			// getTag("<a[^>]*>(?<text>[^<].*?)</a>", content);
			// System.out.println(content);
			// System.out.println(method.getStatusText());
			// k 做为文件名
			int count=1;
		  for(Object xing:XingMingBean.getXings()){
			  for(String ming:XingMingBean.firstMingList){
				  try{ 
					  parse(  xing,  ming,  BASEURL,  baseDir,  pageKey) ;
					  System.out.println(count);
					  count++;
				  }catch(Exception e){
					  e.printStackTrace();
				  }
				  
			  }
		  }

		} catch (Exception e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
	}
    private static String getPageSize(String content){
    	if(content.contains("尾页")){
			String rs=content.substring(content.indexOf("尾页")-5, content.indexOf("尾页")-2);
			if(rs!=null&&rs.trim().length()>0){
				if(rs.contains("=")){
					rs=rs.substring( rs.indexOf("=")+1);
				}
			}
			System.out.println(rs);
			return rs;
		}
    	return "0";
    }
	public static String getInfo(String tmpFile) throws IOException {
		StringBuffer sbFile;
		FileReader in = new FileReader(tmpFile);
		char[] buffer = new char[4096];
		int len;
		sbFile = new StringBuffer();
		while ((len = in.read(buffer)) != -1) {
			String s = new String(buffer, 0, len);
			sbFile.append(s);
		}
		return sbFile.toString();
	}

	 
	private static void parse(Object xing,String ming,String BASEURL,String baseDir,String pageKey) throws IOException {

		  String downKeyValue=xing.toString()+ming;
		  //System.out.println(downKeyValue);
		  String downUrl=BASEURL+downKeyValue;
		  //System.out.println(downUrl);
		  String dir=baseDir+downKeyValue;
		  FileUtil.newFolder(dir);	 
		  DownLoadPagesPay( downUrl, dir+"/"+downKeyValue +".html");
		  File source=new File(dir+"/"+downKeyValue +".html");
		  
		  String content= inputStream2String(new FileInputStream(source));
		  String pageLastNumber = getPageSize( content);
		  int pageNumber=Integer.parseInt(pageLastNumber);
		  if(pageNumber>0)
			for (int i = 2; i <= pageNumber; i++) {
				String pageUrl = downUrl + pageKey + i;
				//System.out.println(pageUrl);
				DownLoadPagesPay(pageUrl, dir+"/"+downKeyValue+i+".html");
			}
		  
	}
	public static String inputStream2String(InputStream is) throws IOException {
		if (is == null) {
			return null;
		}
		BufferedReader in = new BufferedReader(new InputStreamReader(is));
		StringBuffer buffer = new StringBuffer();
		String line = "";
		while ((line = in.readLine()) != null) {
			buffer.append(line);
		}
		return buffer.toString();
	}
	
	
	
	private static int down(){
		Hashtable table=downKeyValue();
		Enumeration en=table.keys();
		int count=0;
		while(en.hasMoreElements()){
			try{
			String key=(String) en.nextElement();
			String url=(String) table.get(key);
			System.out.println(count+","+key+","+url);
			
			if(count>0){
				count++;
				parseOne( url,key);				
			}
			if(key.equals("min.xie")){
				count=1;
			}
			}catch(Exception e){
				e.printStackTrace();
			}
		}
		
		return 0;
	}
	public static String parseOne(String url,String fileName){
		 String baseDir="D:\\";		 	 
		  String dir=baseDir+fileName;
		 // FileUtil.newFolder(dir);	 
		  DownLoadPagesPay( url, baseDir+"/"+fileName +".html");
		 
         return "";
	}
	private static  Hashtable downKeyValue(){
		String BASEURL=" ";		 
		File file = new File("E://");
		Hashtable table=new Hashtable();
		try {
			List<String> list = FileUtils.readLines(file);
			
			for(String url:list){
				if(url!=null){
					String key=url.trim().substring(url.trim().indexOf("=")+1);
					if(!url.startsWith("http")){
						url=BASEURL+url;
					}					
					table.put(key, url.trim());
					
				}
				
			}
			 
			System.out.println( table.size());		 
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}		 
		return table;
	}

}
